{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"THEANO_FLAGS\"]='device=gpu1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ai2-jedi/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n",
      "Using Theano backend.\n",
      "Using gpu device 1: GeForce GTX 980 Ti (CNMeM is disabled, cuDNN Mixed dnn version. The header is from one version, but we link with a different version (4007, 5005))\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math\n",
    "import time\n",
    "import copy\n",
    "from replay_memory import ReplayMemory\n",
    "from grid_world_v2 import GridWorld\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import model_from_json\n",
    "from keras.callbacks import Callback\n",
    "from random import shuffle\n",
    "\n",
    "from keras.layers import MaxPooling2D, Flatten\n",
    "from keras.layers import Convolution2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ------------------------\n",
      "|    .    .    .    .    |\n",
      "|    .    . C1 .    . C2 |\n",
      "|    .    . C3 .    .    |\n",
      "|    .    .    .    .    |\n",
      "|    .    .    .    .    |\n",
      " ------------------------\n"
     ]
    }
   ],
   "source": [
    "grid_world = GridWorld()\n",
    "grid_world.print_grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DQN Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LossHistory(Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.losses = []\n",
    "\n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.losses.append(logs.get('loss'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class DQN(object):\n",
    "    def __init__(self, gamma = 0.975,epsilon = 1, epsilon_decay = .99, terminal_reward = 10, env = None,\n",
    "        memory = None, reward_discount_factor = 0.0, model_name=None, total_cars = 3, grid_size = 6):\n",
    "        \n",
    "        self.env = env\n",
    "        self.losses = []\n",
    "        self.all_games_history = []\n",
    "        self.replay = memory\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = epsilon\n",
    "        self.epsilon_decay = epsilon_decay\n",
    "        self.reward_discount_factor = reward_discount_factor\n",
    "        self.terminal_reward = terminal_reward\n",
    "        self.grid_size = grid_size\n",
    "        self.total_cars = total_cars\n",
    "        self.model = self.loadModel(model_name)\n",
    "    \n",
    "    def loadModel(self,model_name):\n",
    "        if model_name == 'Naive':\n",
    "            model = 'Naive'\n",
    "            return model\n",
    "        else:\n",
    "#             model = Sequential()\n",
    "#             model.add(Dense(256, init='lecun_uniform', input_shape=(self.env.observation_shape,)))\n",
    "#             model.add(Activation('relu'))\n",
    "#             model.add(Dropout(0.2))\n",
    "#             model.add(Dense(256, init='lecun_uniform'))\n",
    "#             model.add(Activation('relu'))\n",
    "#             model.add(Dropout(0.2))\n",
    "#             model.add(Dense(5, init='lecun_uniform'))\n",
    "#             model.add(Activation('linear')) \n",
    "            model = Sequential()\n",
    "            model.add(Convolution2D(32, 3, 3, input_shape=(4, env.grid_size, env.grid_size),activation='relu'))\n",
    "            model.add(Convolution2D(32, 3, 3, activation='relu'))\n",
    "            model.add(Convolution2D(32, 3, 3,activation='relu'))\n",
    "            model.add(Convolution2D(32, 3, 3,activation='relu'))\n",
    "            model.add(Flatten())\n",
    "            model.add(Dense(256, activation='relu'))\n",
    "            model.add(Dense(env.num_actions, init='lecun_uniform'))\n",
    "            model.add(Activation('linear')) \n",
    "    \n",
    "            if model_name != None:\n",
    "                model.load_weights(model_name)\n",
    "                model.compile(loss=\"mse\", optimizer = RMSprop())\n",
    "                return model\n",
    "\n",
    "            else:\n",
    "                model.compile(loss=\"mse\", optimizer = RMSprop())\n",
    "                return model\n",
    "        \n",
    "    \n",
    "    def createTraining(self, minibatch):\n",
    "        X_train = []\n",
    "        y_train = []\n",
    "        for memory in minibatch:\n",
    "            old_state_m, action_m, reward_m, new_state_m,terminal_m = memory\n",
    "            #old_qval = self.model.predict(old_state_m.reshape(1,self.env.observation_shape), batch_size=1)\n",
    "            #newQ = self.model.predict(new_state_m.reshape(1,self.env.observation_shape), batch_size=1)\n",
    "            old_qval = self.model.predict(np.array((old_state_m,)), batch_size=1)\n",
    "            newQ = self.model.predict(np.array((new_state_m,)), batch_size=1)\n",
    "            maxQ = np.max(newQ)\n",
    "\n",
    "            y = np.zeros((1,self.env.num_actions))\n",
    "            y[:] = old_qval[:]\n",
    "\n",
    "            if not terminal_m:\n",
    "                update = (reward_m + (self.gamma * maxQ))\n",
    "            else:\n",
    "                update = reward_m\n",
    "\n",
    "            y[0][action_m] = update\n",
    "\n",
    "            #X_train.append(old_state_m.reshape(self.env.observation_shape,))\n",
    "            X_train.append(old_state_m)\n",
    "            y_train.append(y.reshape(self.env.num_actions,))\n",
    "\n",
    "        X_train = np.array(X_train)\n",
    "        y_train = np.array(y_train)\n",
    "\n",
    "        return X_train,y_train\n",
    "    \n",
    "    def graphLoss(self):\n",
    "        epochs = []\n",
    "        losses_A = []\n",
    "        count = 0\n",
    "        for y in self.losses:\n",
    "            losses_A.append(y[0])\n",
    "            epochs.append(count)\n",
    "            count = count + 1\n",
    "        plt.plot(epochs[:],losses_A[:])\n",
    "        \n",
    "    def train(self, epochs, episodes, max_episode_length,output_weights_name):\n",
    "        for i in range(epochs + 1):\n",
    "            for j in range(episodes):\n",
    "                terminal = False\n",
    "                self.env.reset()\n",
    "                \n",
    "                while(not terminal and self.env.t < max_episode_length):\n",
    "                    self.env.t += 1\n",
    "                    print (\"Epoch #: %s\" % (i,))\n",
    "                    print(\"Game #: %s\" % (j,))\n",
    "                    print (\"Timestep #: %s\" % (self.env.t,))\n",
    "                    print ('Epsilon : ', float(self.epsilon))\n",
    "                    self.env.print_grid()\n",
    "\n",
    "                    #Bookeeping\n",
    "                    curr_history = {}\n",
    "                    curr_history['t'] = self.env.t\n",
    "                    curr_history['curr_state'] = (self.env.cars_grid.copy(), self.env.cust_grid.copy())  \n",
    "\n",
    "                    #Step all agents 1 timestep\n",
    "                    all_agents_step = self.env.stepAll(self.model,self.epsilon)\n",
    "                    \n",
    "                    #More Bookeeping\n",
    "                    curr_history['actions'] = all_agents_step\n",
    "                    curr_history['next_state'] = (self.env.cars_grid.copy(), self.env.cust_grid.copy())\n",
    "                    self.env.history.append(curr_history)  \n",
    "\n",
    "                    #Check if terminal amount of cutomers picked up to end episode\n",
    "                    terminal = self.env.isTerminal()\n",
    "                    clear_output(wait=True)\n",
    "                    \n",
    "                    #Add memory to replay\n",
    "                    for memory in all_agents_step:\n",
    "                        self.replay.addToMemory(memory, terminal)\n",
    "\n",
    "                #After each game/episode fit the network with a minibatch sampled from the replay memory\n",
    "                if self.replay.isFull():\n",
    "                    minibatch = self.replay.getMinibatch()\n",
    "                    X_train, y_train = self.createTraining(minibatch)  \n",
    "                    history = LossHistory()\n",
    "                    self.model.fit(X_train, y_train, batch_size = self.replay.batchSize, nb_epoch=1, verbose=1,callbacks=[history])\n",
    "                    self.losses.append(history.losses)\n",
    "\n",
    "                #Append Bookept Hostory\n",
    "                self.all_games_history.append(self.env.history)\n",
    "\n",
    "            #Exponentially decay epsilon and run test metrics\n",
    "            if self.replay.isFull():\n",
    "                self.epsilon = self.epsilon * self.epsilon_decay\n",
    "                if np.mod(i,20) == 0:\n",
    "                    self.model.save_weights('weights/' + output_weights_name + '_'+ str(i) +'.h5', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "obstacle_grid = np.array([[0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,0,0,0,0,0,0,0],\n",
    "                          [0,0,0,0,0,0,0,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "memory = ReplayMemory(buffer = 50000, batchSize = 500)\n",
    "env = GridWorld(num_cars = 2, grid_size = 10, terminal_reward = 5, demand_limit = 2, obstacle_grid = obstacle_grid)\n",
    "dqn = DQN(memory = memory, env = env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "500/500 [==============================] - 0s - loss: 0.0059\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "episodes = 30\n",
    "max_episode_length = 50\n",
    "output_weights_name = 'Gridworld_6x6_4Layer_Conv'\n",
    "\n",
    "dqn.train(epochs, episodes, max_episode_length, output_weights_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEACAYAAACznAEdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHw5JREFUeJzt3XuUFOWZx/Hvg2gUL2jcqCsoGi8xoglqRLyF2YiCmg1J\nlEQ3iZddE4yirKsG1E0gZPccPTHelkQkGvEShVVjRA8q3kbdBAUiiMAAoygCKoYgICgwzDz7x1uV\nvkzPdPdMT1cN8/ucU6fr8tZbz1R319Nv1Vs15u6IiIh0SzoAERFJByUEEREBlBBERCSihCAiIoAS\ngoiIRJQQREQEKDEhmNkQM1tkZkvMbFSB5V8wsz+b2SYz+49y1hURkXSwYvchmFk3YAlwCvAeMAs4\nx90XZZX5B6AP8E3gI3e/qdR1RUQkHUppIfQH6t19mbs3AJOBodkF3H21u/8F2FruuiIikg6lJIRe\nwPKs6RXRvFK0Z10REakiXVQWEREAupdQZiWwf9Z072heKUpe18z0UCURkTK5u1WqrlJaCLOAg82s\nj5ntAJwDTG2lfHZwZa3r7qxb57inYxgzZkziMSimbSemtMalmDpvTJVWtIXg7o1mNgKYTkggd7l7\nnZkND4t9opntDcwGdgWazGwkcLi7byi0bkvbmj0bjj0WOuDvFBGRIko5ZYS7PwV8IW/eHVnjq4D9\nSl23JX/7WymlRESkI+iicitqamqSDqEZxVSaNMYE6YxLMZUmjTFVWtEb06rFzPypp5whQ3TKSESk\nFGaGV/misoiIdAFKCCIiAighiIhIRAlBRESAlCUEq9ilERERKVeqEoKIiCRHCUFERAAlBBERiSgh\niIgIoIQgIiIRJQQREQFSlhDU7VREJDmpSgh6qJ2ISHJSlRBERCQ5SggiIgIoIYiISEQJQUREACUE\nERGJpCohqNupiEhyUpUQREQkOUoIIiICKCGIiEhECUFERAAlBBERiSghiIgIkLKEoG6nIiLJSVVC\nEBGR5CghiIgIkLKEoP+HICKSnFQlBBERSY4SgoiIAClLCOplJCKSnFQlBBERSY4SgoiIACUmBDMb\nYmaLzGyJmY1qocxtZlZvZnPNrF/W/CvMbL6ZzTOz35vZDpUKXkREKqdoQjCzbsB4YDDQFzjXzA7L\nK3M6cJC7HwIMByZE8/cFLgOOdvcvAd2Bcyr6F4iISEWU0kLoD9S7+zJ3bwAmA0PzygwF7gVw91eB\nnma2d7RsO2BnM+sO9ADeq0jkIiJSUaUkhF7A8qzpFdG81sqsBHq5+3vAr4B3o3lr3f3ZtocrIiId\npXtHVm5muxNaD32AdcDDZvYv7v5AofL33jsWgLFjoaamhpqamo4MT0SkU6mtraW2trbD6jcv8rwI\nMxsAjHX3IdH0aMDd/YasMhOAF9x9SjS9CBgInAwMdvcfRvN/ABzn7iMKbMeffdYZNEiPsBARKYWZ\n4e4Vu4OrlFNGs4CDzaxP1EPoHGBqXpmpwHlRgAMIp4ZWEU4VDTCzHc3MgFOAukoFLyIilVP0lJG7\nN5rZCGA6IYHc5e51ZjY8LPaJ7j7NzM4wszeBjcCF0bozzexhYA7QEL1O7Kg/RkRE2q7oKaNq0Skj\nEZHyJHHKqOoOPTTpCEREup5UJoT6+qQjEBHpelKZEEREpPqUEEREBFBCEBGRiBKCiIgASggiIhJR\nQhAREUAJQUREIqlKCFax++1ERKRcqUoIIiKSHCUEEREBlBBERCSihCAiIoASgoiIRJQQREQEUEIQ\nEZGIEoKIiABKCCIiElFCEBERQAlBREQiSggiIgIoIYiISCRVCUFPOxURSU6qEoKIiCRHCUFERAAl\nBBERiSghiIgIoIQgIiIRJQQREQFSlhDU7VREJDmpSggiIpKcVCUE96QjEBHpulKVEEREJDlKCCIi\nApSYEMxsiJktMrMlZjaqhTK3mVm9mc01s35Z83ua2UNmVmdmC8zsuEoFLyIilVM0IZhZN2A8MBjo\nC5xrZofllTkdOMjdDwGGAxOyFt8KTHP3LwJfBuoqFLuIiFRQKS2E/kC9uy9z9wZgMjA0r8xQ4F4A\nd38V6Glme5vZbsDJ7n53tGyru69vaUPqdioikpxSEkIvYHnW9IpoXmtlVkbzDgRWm9ndZvaamU00\ns53aE7CIiHSMjr6o3B04Gvi1ux8NfAKM7uBtiohIG3QvocxKYP+s6d7RvPwy+7VQZrm7z47GHwYK\nXpQGmDRp7N/Ha2trqKmpKSE8EZGuoba2ltra2g6r37zI3WBmth2wGDgFeB+YCZzr7nVZZc4ALnX3\nM81sAHCLuw+Ilr0I/NDdl5jZGKCHuzdLCmbmL7zg/NM/hWndpCYi0jozw90rdvW1aAvB3RvNbAQw\nnXCK6S53rzOz4WGxT3T3aWZ2hpm9CWwELsyq4nLg92a2PbA0b5mIiKRE0RZCtaiFICJSnkq3EFJ1\np7K6nYqIJCdVCSHbOeckHYGISNeS2oQwZUrSEYiIdC2pTQgiIlJdqUoIupAsIpKcVCUEERFJjhKC\niIgAKUsI2d1O1QVVRKS6UpUQbrwx6QhERLquVN2pDLmxpCQ0EZFU2qbvVBYRkeSkNiHoGoKISHWl\nNiGIiEh1KSGIiAjQCRLCK6/o9JGISDWkNiHEPYzq6lovJyIilZHahCAiItWlhCAiIoASgoiIRFKf\nEHS3sohIdaQ+IYiISHUoIYiICKCEICIikdQnBN2UJiJSHalPCCIiUh2pTwjqZSQiUh2pTwgiIlId\nSggiIgKkPCF8+GHSEYiIdB2pTgibNycdgYhI15HqhCAiItWjhCAiIkDKE4JuShMRqZ5UJwTdgyAi\nUj2pTggiIlI9qU4IZmoliIhUS0kJwcyGmNkiM1tiZqNaKHObmdWb2Vwz65e3rJuZvWZmU8sJ7pNP\nyiktIiLtUTQhmFk3YDwwGOgLnGtmh+WVOR04yN0PAYYDE/KqGQksLDe4W24pdw0REWmrUloI/YF6\nd1/m7g3AZGBoXpmhwL0A7v4q0NPM9gYws97AGcCd5Qa3ZUu5a4iISFuVkhB6AcuzpldE81orszKr\nzM3A1UDZVwOamspdQ0RE2qpDLyqb2ZnAKnefC1g0lKyxUfciiIhUS/cSyqwE9s+a7h3Nyy+zX4Ey\nZwPfMLMzgJ2AXc3sXnc/r/CmxmaN12BWo15GIiKR2tpaamtrO6x+8yJHXDPbDlgMnAK8D8wEznX3\nuqwyZwCXuvuZZjYAuMXdB+TVMxC40t2/0cJ2PP+s0gUXwEknwUUXqfupiEg+M8PdK3YepWgLwd0b\nzWwEMJ1wiukud68zs+FhsU9092lmdoaZvQlsBC6sRHCTJsGee1aiJhERKaZoC6FaCrUQsqUkTBGR\n1Kh0CyHVdyqLiEj1KCGIiAighCAiIhElBBERAZQQREQkooQgIiKAEoKIiESUEEREBFBCEBGRiBKC\niIgASggiIhJRQhAREUAJQUREIkoIIiICKCGIiEhECUFERAAlBBERiSghiIgIoIQgIiIRJQQREQGU\nEEREJKKEICIigBKCiIhElBBERARQQhARkYgSgoiIAJ0oISxdmnQEIiLbtk6TEK68EhoaWl7+7rvg\nXr14RES2NZ0mIfzxj/DWWy0v79MHXn65evGIiGxrOk1CgOItgI8/rk4cIiLbok6VEArZsAFGjgzj\njz8OZsnGIyLSWZmn5MS7mTm0HsuCBXD44bnz/vQnOOmk3Hkp+ZNERDqUmeHuFfsZ3KlaCIUO9GoR\niIhURqdKCOVSS0FEpHSdKiE8/njzeYVaCL/6VXjt1i2cZhIRkeI6VUK46abm8wolhKuuyox/8EHH\nxSMisi0pKSGY2RAzW2RmS8xsVAtlbjOzejOba2b9onm9zex5M1tgZm+Y2eXtCbacU0AXXNCeLYmI\ndD1FE4KZdQPGA4OBvsC5ZnZYXpnTgYPc/RBgODAhWrQV+A937wscD1yav262e+5pPZZyLirHdQ0a\nBFOmtF6viIiU1kLoD9S7+zJ3bwAmA0PzygwF7gVw91eBnma2t7t/4O5zo/kbgDqgV0sbOu+88i8E\nl9LLSHcwi4gUV0pC6AUsz5peQfODen6ZlfllzOwAoB/warlBxvKTxZYtcPfd5a8nIiLNda/GRsxs\nF+BhYGTUUiho7NixWVM10ZCxZk1mvFcv+PKX4ckni29fCUFEtgW1tbXU1tZ2WP1F71Q2swHAWHcf\nEk2PBtzdb8gqMwF4wd2nRNOLgIHuvsrMugNPAE+6+62tbMfjWFo7DRSHW84NacOHw4QJxcuJiHQm\nSdypPAs42Mz6mNkOwDnA1LwyU4HzogAHAGvdfVW07HfAwtaSQblWrSpeJptaCCIixRVNCO7eCIwA\npgMLgMnuXmdmw83sR1GZacDbZvYmcAfwYwAzOxH4HvA1M5tjZq+Z2ZBi21y8uPXl5V4kLichmMF9\n95VXv4jItiBVD7eLY/nkE9h558Ll3OGRR+Dss0uv+9/+De68s9Q4YLvtYOvW0usXEUlCl3i4XbHr\nA93KjPrtt2HGjNbrfeONcK1BRKSrSmVC6N5K36fVq8Mv+HLMnAmLFoVx99BbadYs2HXXTJkHH4SJ\nE8uPVURkW5HKhLD99i0vO/vs8h95vWED/Ou/hvFf/xr23BNGjAjzAY49Fj76KFNej9QWka4olQkB\n4H//t/D8F18s/5RRtssuC68zZ4bXxkaYPRuWLGl7nSIi24LUJoTWtCch5ItPTzU1ZeZt3Qq/+U0Y\nf/JJGDeuctsTEUmrLp8QYvmdrS69NLz+13/BmDGV356ISNp0yoTQEef4X3yx8nWKiHQmqU0I8UG/\n0E1qgwdXJ4Zx42D9+upsS0QkaVV5uF17HHpoctvWqSIR6UpS20IQEZHqUkIQERFACaFku+0G8+eX\nt86ZZ8Jxx3VMPCIildYpEsIvfpF0BPDxx/D66/D++7B8efHyANOmZW6AExFJu1Q+7RTg4Ydh2LBw\nf4B7x9x70F4ffwy77NLy8rinVPYuXrcuzN9tt46NTUS2fV3iaaf50vpsofXrYfx4aGiAq68Oj+1u\nyfvvh8dkHHkknHBC4TKltjxERDpCp0gIAM8/n3QEzW3ZEp6NNGMG3Hhj7jWGu+/OLbvvvvCzn4WD\n/oIFcNVVucu3boX99+/4mEVEWpLaU0bxP8HJDi+tLYXYjBkwYEAYz47VvXDsDz0EZ50VljU0wA47\nhGcqFSp74YXwve/BoEEdE7uIdD5d5pTRySfDBRckHUV53GHz5nAxOVufPoXLDxsWTjn99KewYkWY\n9+1vwzXXZMrcfHN4NPekSZlHeJcTT2Njeevk+/TT3EeDi8i2K7UthMJlqhRMG730EixcCBdfXP66\n//M/mUdz77svrFyZaTUMHJh51tLTT8Opp5a2L26/HS65JHNh/r33oFev8uIaNixc4E/Jx6SoN94I\n12lEuoIu00LojL761bYlA8gkAwgH32eeCckAwn+Jiw0eHP65T/wf4MxyH92dLb6mMX9+OAXXuzfM\nmVNeq2HZstaXr14NGzeWXl/s9ddh7dry14udeGLYD/m+9CVYtart9bakoSE8+VZkm+buqRhCKK3L\n/NZtfTjllNLLpnHo3j13etddm5cZNsz9618P47Nmuc+fH/bRRx+5L1zofvrpueUnTsyMT56cu1+v\nv95969bC+7tnz/D67W+7z5uXWTZtmvuaNWHZkCHN173//hDLM8+4z5hRuO4f/MB9yhT3DRvCvIYG\n9x13zJSZN8/9xRfdN24M9TQ0uL/9dngF9/33d9+8uXm9K1YU/SiVrb4+1J0/b9my9tX7ySdh2NaM\nGuX+la8kHcW2LzpuVu44XMnK2hVIBRPC5MmZ8dra5A/w1Rp+8pOWl/3yl5nxu+5y/8tfwvjWreH1\nT39yf+IJ95NOcn/jjcL7+7//O8zfsKH5skMOCUni/vvdf/GLMO873wmve+zhfvnl7j/+sfs//7P7\nVVflrvvDH4Z66+r870nI3f2gg8L0iBHhdfz45u8vuN96q/vixZkE9dBD7uvWua9dG+rZbjv3N9/M\n/Sy9/XZIpO7uEyaEg3u2eL/E42+9lZnetMn99tvD9D/+Y9GPbY7f/tb9nXcy08cc437EEbllxoxx\nf/fd8uqtpDVr2l/HEUdk9ldbNTS4DxzY/lh69XJvamp/PW1x3XXu69d3XP1KCC0MS5ZkxrMPGMXW\n66rDgQe2vvzZZ5vPO+ss9xNOcP/a1yofz/bb506PGdO8jFl4HTKkcB2HHpoZ/8IXct//XXd1HzfO\n/bHH3C+7rPDnY8uWzGft00/DvOefD68LF2bKT5mSu92PPir60c35DO+wg/sf/hCm49Zgfplf/rL5\numvWuP/7v5e+rbaCwgexTZtKr6Nv3+Z/V7niBN8eTU2hjsbG9tXTVuD+1FPuzz3XUfXj7koIzYY4\nIWzaFH79FfrCa+jaQ69ezecNHJgZHz/e/eqr3adPz8y7++7c8qec4v75z+fOC/0hgk2bwsHn6adb\nPg0XD926Zcbd3b/1rUziOeoo91deCYlhxIiwfOrUsOy669z32y+06FauzK1/4UL3118v+lXyO+4I\nZeMD5W9+E5JTHOOf/+z+wAPNY3/88dbrbWoKLcyWEsKYMYVPIRYS7/v2aGwMdRR6L/KddVbLLYnV\nq3OTysiR7i+95P7hh7nlIHMKNJ6eNi28tvf0YiFdOiGMG9fylz1OCLH4VJF7OP/c1OS+fLn7tde6\n9++f/MFJw7Y1fO5z7vfeG8bjA8DYseHU1d57Zw6kLa0fX/e64ILCyw88MJMQ8oeJEzMHsh13DPNm\nzQrbnzMn/Nq/+eb8A0kYrrkmfHfiJBe3jL761fA6blw4gMe/tMH9nnty6/rMZ9zvuy+0lOLrWocf\nHl4XLAiv8fWnuI5hw8J1BvdQ3333hW3PmOF+5JGhrrhsY2NoUR15ZEh2V1wRvscffph7kF63LpQ/\n/fQw/tJL4UAO4fRTtldeCYn/gQfCcMABodyMGe6f/WzYf6tXh6S5dGlY9v3vu7/wQkgG2fs/PsUZ\n/33Ll+dOZ79vs2aFfXT++UUPdyXp0gnh5z+PIo4inzw5fDDiD80jj2TKzpqVKZvv1FNz39Arrwyv\n8XljDRo6Ypg7t33rFzvN19KQf/rtsMPKryO/VfR//xcSzd/+1vp6++yTGb/44uLbib+L2UP+9zV/\nKHTt7Pvfz53effeQlJ57LrTEWqor+/rWaaeVvn8efTS02MD97LPDda34h8GkSZly8bUnKHq4K0mX\nTgjXX5/ZkRDO5T76aMs7t6UmWv4bfccdmTryWyHvvJP51aVBgwYNlRrq6ooe8oqqdELoVPchjBwZ\n+tFnO+II2GmnwuVbejbQ7rvnTm/Zkhn/6U8z4wsWhLuM88uLiGyLOlVC2HFH6NcvjA8aBMcfDwcf\n3PpTRguZOBHq6zPT+Y+ivvhiGDcODj88TN95Z0hGP/tZpsymTa1v49xzm8/r0aPtD7D74hfbtp6I\npNN22yUdQQGVbG60ZwihVFfcdGtsdH/vvdLWWbs2rLN1a+gW98ILmXrWrQs9Ti65JJR99NHQTz3u\nWrhxY5gf96cvdcj2zjth3v33Zy7YddSw227JNadvvtl9r73C+LXXuj/4YHKxaNDQEUP+vTFtO4bh\n7l30GkKlffe70R4oQ9yTIbt72gknhJ5L7qGnRHYXt8bG0MMhvpPYPVxwgtDlcY89wnjcuwNC74iF\nC1vu9x3fUOWe+wF76aXM+PHHt/xBrKnJjK9f7/7++7kX7k44IbzGyS8eWuoBM2VK5s7oZ58NiXDW\nLPfRo8O8xx4LCbeleI46Krxu3Bj2Qyz72o576DETr5PdC6Utw5135k4/9FD76tOgodzhtdfKO/YU\nooRQQRs3Nu9HXEycELJt2ZJ7U1MxS5aEZOQeutvddlsYjy9oz55del0Q7rR95ZXMvBNPDAfM+CD8\n5JPuPXq433RT7l2x+dtpasokoTvuCNPxzV5nnx3mNzS4v/xy6Ea5dWum1TNtWrgbOV9+dz/3kIQg\n9KGP74ou9Gupqan5DVL587p3D713Vq7MfMlmzgxDbW2mX//y5ZnHdyxdmtnmp5+6//WvYXru3NB7\nBjJ/16RJoVNBnGzj1t6qVSGWt9/OJPV46NEjPOpj9uwwHfd4i4dLLgmvO+8cXrNbmWkffvSjlped\neWbr655/fsfGNmyY+0UX5c7L7x3VluHYY3Pf21LXGzWq9eXl3OjXEiWEhG3aFO21DjJzZnm32X/8\nccfFEtu4sfkzg9qrnARaqpbuGI4TQKmgcCJzD/s7v/fali3hPoLs/ufZ4j78P/95KFso0X3zm+EO\n8M2b3V99NSyfNy8zfOc74cfIkiXuN9wQbhybNy/ctNbUFJJb9o13I0aEHxoTJmRar3V1odU3c2Z4\nfhKEewAKHaz69s39HDY1ZR4x0tSUu+z5590XLQrjDzyQW8/o0bmPBVm8OOzD+PEmo0eH51U98UTo\nEgru/fqFRBtbtCj8rR9/HLqM3nhj+FsbGkIcS5fmxnPggaFLev5Ne6tWhbLnnRdaiJs2hTLZrcN9\n9w0t0fr6cHo2brGuXZv7OJHnngvb/OijTEv2mGMy9cR3Jm/e7P6732XuZfj61zM/iCrxOI1KJ4RO\n9fhrEWnZ1q2wZk34nxz77Ve8/JIlcOihYXzhwtC5Yp99wqPXd9oJ9tqrY+NdsSI86j37/6WvX5/M\n/xtvaqru/22fPz/0kGyvSj/+WglBRKSTSuT/IZjZEDNbZGZLzGxUC2VuM7N6M5trZv3KWVdERJJX\nNCGYWTdgPDAY6Auca2aH5ZU5HTjI3Q8BhgMTSl03zWpra5MOoRnFVJo0xgTpjEsxlSaNMVVaKS2E\n/kC9uy9z9wZgMjA0r8xQ4F4Ad38V6Glme5e4bmql8QOgmEqTxpggnXEpptKkMaZKKyUh9AKWZ02v\niOaVUqaUdUVEJAU66rp6xS5yiIhIdRTtZWRmA4Cx7j4kmh5N6Pt6Q1aZCcAL7j4lml4EDAQOLLZu\nVh3qYiQiUqZK9jLqXkKZWcDBZtYHeB84B8h/dNtU4FJgSpRA1rr7KjNbXcK6QGX/KBERKV/RhODu\njWY2AphOOMV0l7vXmdnwsNgnuvs0MzvDzN4ENgIXtrZuh/01IiLSZqm5MU1ERJKV+P9DqOaNa2Z2\nl5mtMrN5WfP2MLPpZrbYzJ42s55Zy66JbrarM7PTsuYfbWbzophvaWdMvc3seTNbYGZvmNnlScdl\nZp8xs1fNbE4U05ikY8qqr5uZvWZmU1MU0ztm9nq0v2amIS4z62lmD0XbWGBmxyX8mTo02j+vRa/r\nzOzyFOynK8xsflTf781shxTENDL63lX/eFDJByOVOxAS0ptAH2B7YC5wWAdu7ySgHzAva94NwE+i\n8VHA9dH44cAcwmm1A6I44xbVq8Cx0fg0YHA7YtoH6BeN7wIsBg5LQVw9otftgFcI95QkGlNUxxXA\n/cDUNLx/UR1LgT3y5iX9/k0CLozGuwM9k44pK7ZuwHvAfknGBOwbvXc7RNNTgPMTjqkvMA/4DOG7\nNx04qFoxteuNrcAHYwDwZNb0aGBUB2+zD7kJYRGwdzS+D7CoUCzAk8BxUZmFWfPPAW6vYHx/BAal\nJS6gBzAbODbpmIDewDNADZmEkPh+At4G9sybl1hcwG7AWwXmJ76vonpOA15OOiZCQlgG7EE4oE5N\n+rsHnA38Nmv6P4GrgbpqxJT0KaM03Li2l7uvAnD3D4D4GY/5sa0kc7Pdiqz5FYvZzA4gtGBeIbz5\nicUVnZqZA3wAPOPus5KOCbiZ8OXIvvCVdExE8TxjZrPM7KIUxHUgsNrM7o5O0Uw0sx4Jx5Ttu8AD\n0XhiMbn7e8CvgHej+te5+7NJxgTMB06OThH1AM4gtKSqElPSCSGNErnKbma7AA8DI919Q4E4qhqX\nuze5+1GEX+X9zaxvkjGZ2ZnAKnefS+s3Pibx/p3o7kcTvryXmtnJBeKoZlzdgaOBX0dxbST8kkz0\nMwVgZtsD3wAeaiGGan6mdic8SqcPobWws5l9L8mY3H0R4fTQM4TTPHOAxkJFO2L7SSeElUD2v53v\nHc2rplUWnruEme0DfJgVW/ZT5ePYWprfZmbWnZAM7nP3x9ISF4C7rwdqgSEJx3Qi8A0zWwo8CHzN\nzO4DPkh6P7n7+9HrXwmn/PqT7L5aASx399nR9COEBJGGz9TpwF/cfXU0nWRMg4Cl7r7G3RuBR4ET\nEo4Jd7/b3b/i7jXAWsJ1xarElHRC+PtNb2a2A+E819QO3qaR+wtzKnBBNH4+8FjW/HOiXgcHAgcD\nM6Pm2joz629mBpyXtU5b/Y5wvu/WNMRlZv8Q92Iws52AUwnnMBOLyd2vdff93f3zhM/J8+7+A+Dx\npGICMLMeUesOM9uZcH78DZLdV6uA5WYW/fsbTgEWJBlTlnMJCT2WZEzvAgPMbMeorlOAhQnHhJl9\nLnrdH/gW4fRadWJq7wWi9g6EX56LgXpgdAdv6wFC74bNhA/DhYQLSs9GMUwHds8qfw3hqn0dcFrW\n/GMIX/p64NZ2xnQioUk4l9A8fC3aJ59NKi7gyCiOuYQeD9dF8xOLKS++gWQuKicaE+F8ffzevRF/\nhlMQ15cJP7jmAn8g9DJKOqYewF+BXbPmJR3TmKj+ecA9hN6OScf0EuFawhygppr7STemiYgIkPwp\nIxERSQklBBERAZQQREQkooQgIiKAEoKIiESUEEREBFBCEBGRiBKCiIgA8P/S4cmc3yJzZgAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7efc51587890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dqn.graphLoss()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
